# -*- coding: utf-8 -*-
import os
import re
import json
import time
import scrapy
from traceback import format_exc
from scrapy.selector import Selector
from data_acqr.items import DataAcqrItem


class JdSpider(scrapy.Spider):
    name = 'jd'
    allowed_domains = ['search.jd.com', 'item.jd.com', 'club.jd.com']
    start_urls = ['https://search.jd.com/Search?keyword=气泡水&enc=utf-8&wq=气泡水&pvid=31e69910f9a0466aa859c0bbe5e10f55']

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        content_list = Selector(response).xpath('//div[@id="J_goodsList"]/ul/li')[:10]
        for content in content_list:
            item = DataAcqrItem()
            item['product_link'] = ''.join(['http:', content.xpath('./div/div/a/@href').extract_first()])
            yield scrapy.Request(item['product_link'], meta={'item': item}, callback=self.parse_product_detail)

    def parse_product_detail(self, response):
        item = response.meta['item']
        item['product_name'] = Selector(response).xpath('//div[@class="detail"]/div/div[2]/div/div/ul[2]/li[1]/text()'
                                                        ).extract_first().split('商品名称：')[1]
        item['product_id'] = Selector(response).xpath('//div[@class="detail"]/div/div[2]/div/div/ul[2]/li[2]/text()'
                                                      ).extract_first().split('商品编号：')[1]
        for page in range(0, 10):
            comment_url = f'https://club.jd.com/comment/productPageComments.action?callback=&' \
                              f'productId={item["product_id"]}&score=0&sortType=6&page={page}&' \
                              f'pageSize=10&isShadowSku=0&rid=0&fold=1'
            yield scrapy.Request(comment_url, meta={'item': item}, callback=self.parse_comments)


    def parse_comments(self, response):
        try:
            content = json.loads(response.text)
            if 'comments' not in content or not content['comments']:
                raise Exception('No comments')
            for comment_info in content['comments']:
                item = response.meta['item']
                item['comment'] = re.sub('\s+', '', comment_info['content'])
                item['comment_tm'] = comment_info['creationTime']
                item['comment_star'] = comment_info['score']
                yield item
        except Exception as e:
            print(response.text)
